
import requests
from bs4 import BeautifulSoup
import pandas as pd
import datetime as dt
import re
import string


import plotly.express as px


url = "https://en.wikipedia.org/wiki/List_of_Helsinki_Metro_stations"


headers={'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0',
        "Accept": "*/*",
"Accept-Language": "en-US,en;q=0.5",
"Accept-Encoding": "gzip, deflate, br",
"Origin": "http://www.wikipedia.org",
"Connection": "keep-alive"
}


response=requests.get(url,headers=headers)


soup=BeautifulSoup(response.content,"html.parser")
#print(soup.prettify()[0:1000])


a = soup.select(".mw-parser-output")


items = soup.find("table", attrs={"class":"wikitable sortable"})


#print(items.prettify())


stations = []
for station in items("a"):
    if "Tram" not in station.text:
        if "rail" not in station.text:
            stations.append(station.text)


stations

['Aalto-yliopisto',
 'Hakaniemi',
 'Herttoniemi',
 'Itäkeskus',
 'Helsingin yliopisto',
 'Kalasatama',
 'Kamppi',
 'Keilaniemi',
 'Koivusaari',
 'Kontula',
 'Kulosaari',
 'Lauttasaari',
 'Matinkylä',
 'Mellunmäki',
 'Myllypuro',
 'Niittykumpu',
 'Puotila',
 'Rautatientori',
 'Rastila',
 'Ruoholahti',
 'Siilitie',
 'Sörnäinen',
 'Tapiola',
 'Urheilupuisto',
 'Vuosaari']


types = []
td = items.find_all("td")
for n in range(3, len(td),4):
    types.append(td[n].text[:-1])
#types


# for station in items("a"):
#     if "Tram" not in station.text:
#         if "rail" not in station.text:
#             print(station)


urls=[]
for station in items("a"):
    if "Tram" not in station.text:
        if "rail" not in station.text:
            url = "https://en.wikipedia.org/" + station["href"]
            urls.append(url)


urls

['https://en.wikipedia.org//wiki/Aalto_University_metro_station',
 'https://en.wikipedia.org//wiki/Hakaniemi_metro_station',
 'https://en.wikipedia.org//wiki/Herttoniemi_metro_station',
 'https://en.wikipedia.org//wiki/It%C3%A4keskus_metro_station',
 'https://en.wikipedia.org//wiki/University_of_Helsinki_metro_station',
 'https://en.wikipedia.org//wiki/Kalasatama_metro_station',
 'https://en.wikipedia.org//wiki/Kamppi_metro_station',
 'https://en.wikipedia.org//wiki/Keilaniemi_metro_station',
 'https://en.wikipedia.org//wiki/Koivusaari_metro_station',
 'https://en.wikipedia.org//wiki/Kontula_metro_station',
 'https://en.wikipedia.org//wiki/Kulosaari_metro_station',
 'https://en.wikipedia.org//wiki/Lauttasaari_metro_station',
 'https://en.wikipedia.org//wiki/Matinkyl%C3%A4_metro_station',
 'https://en.wikipedia.org//wiki/Mellunm%C3%A4ki_metro_station',
 'https://en.wikipedia.org//wiki/Myllypuro_metro_station',
 'https://en.wikipedia.org//wiki/Niittykumpu_metro_station',
 'https://en.wikipedia.org//wiki/Puotila_metro_station',
 'https://en.wikipedia.org//wiki/Rautatientori_metro_station',
 'https://en.wikipedia.org//wiki/Rastila_metro_station',
 'https://en.wikipedia.org//wiki/Ruoholahti_metro_station',
 'https://en.wikipedia.org//wiki/Siilitie_metro_station',
 'https://en.wikipedia.org//wiki/S%C3%B6rn%C3%A4inen_metro_station',
 'https://en.wikipedia.org//wiki/Tapiola_metro_station',
 'https://en.wikipedia.org//wiki/Urheilupuisto_metro_station',
 'https://en.wikipedia.org//wiki/Vuosaari_metro_station']


ominaisuudet = ["Coordinates", "Platforms", "Tracks", "Depth", "Fare zone", "Opened"]


def station_exploration(url):
    
    stationParameters = {}
    
    response = requests.get(url)
    soup = BeautifulSoup(response.text,"html.parser")
    
    nimi = soup.find("title")
    nimi = nimi.text.replace(" metro station - Wikipedia", "")
    stationParameters["Name"]=nimi
    
    infobox = soup.find("table",attrs={"class":"infobox vcard"})
    infos = infobox.find_all("tr")
    
    for i in infos:
        if i.th !=None and i.td != None:
            if i.th.text in ominaisuudet:
                stationParameters[i.th.text] = i.td.text 
    
    for i in infos:
        if i.td != None and "daily" in i.td.text:
            #print(i.td.text)
            stationParameters["Passengers"]=i.td.text
    
    return stationParameters


def parametersCleaning(stationParameters):
    
    if 'Coordinates' in stationParameters.keys():
        stationParameters['Coordinates'] = stationParameters['Coordinates'][-18:]
        
        coord = []
        osat = stationParameters['Coordinates'].split(";")
        
        for alkio in osat:
            alkio = alkio.strip()
            alkio = re.sub(r'.+(?=60)' , "", alkio)
            alkio = float(alkio)
            coord.append(alkio)

            
        stationParameters['Coordinates'] = coord   
        
    
    if 'Platforms' in stationParameters.keys():
        if stationParameters['Platforms']=="Island platform":
            stationParameters['Platforms'] = 1
        else:
            stationParameters['Platforms'] = int(stationParameters['Platforms'])      
    
    stationParameters['Tracks'] = int(stationParameters['Tracks'])
    
    
    if 'Depth' in stationParameters.keys():
        stationParameters['Depth'] = stationParameters['Depth'][:3]
        stationParameters['Depth'] = int(stationParameters['Depth'])
    else:
        pass
    
    if "A" in stationParameters['Fare zone']:
        stationParameters['Fare zone'] = "A"
    else:
        stationParameters['Fare zone'] = "B"
        
    stationParameters['Opened'] = dt.datetime.strptime(stationParameters['Opened'], "%d %B %Y")
    
    stationParameters['Passengers'] = re.sub(r'\[\w+\]', '', stationParameters['Passengers'])
    stationParameters['Passengers'] = stationParameters['Passengers'].replace(" daily", "").replace(",", "") 
    
    
    stationParameters['Passengers'] = int(stationParameters['Passengers'])
    
    return stationParameters


# parsing through stations
luettelo = {}

for i in range(len(stations)):
    #print(stations[i])
    stationParameters = station_exploration(urls[i])
    
    stationParameters = parametersCleaning(stationParameters)
    luettelo[stations[i]] = stationParameters


i = 0
for key in luettelo:
    luettelo[key]["Type"] = types[i]
    i +=1


luettelo

{'Aalto-yliopisto': {'Name': 'Aalto University',
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 20,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 14500,
  'Type': 'Underground'},
 'Hakaniemi': {'Name': 'Hakaniemi',
  'Coordinates': [60.18, 24.95056],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 23,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1982, 6, 1, 0, 0),
  'Passengers': 28100,
  'Type': 'Underground'},
 'Herttoniemi': {'Name': 'Herttoniemi',
  'Coordinates': [60.195, 25.03083],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1982, 6, 1, 0, 0),
  'Passengers': 34500,
  'Type': 'At-grade'},
 'Itäkeskus': {'Name': 'Itäkeskus',
  'Coordinates': [60.21028, 25.07806],
  'Platforms': 2,
  'Tracks': 3,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1982, 6, 1, 0, 0),
  'Passengers': 40600,
  'Type': 'At-grade'},
 'Helsingin yliopisto': {'Name': 'University of Helsinki',
  'Coordinates': [60.17222, 24.9475],
  'Tracks': 2,
  'Depth': 27,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1995, 3, 1, 0, 0),
  'Passengers': 24000,
  'Type': 'Underground'},
 'Kalasatama': {'Name': 'Kalasatama',
  'Coordinates': [60.1875, 24.97694],
  'Platforms': 2,
  'Tracks': 2,
  'Fare zone': 'A',
  'Opened': datetime.datetime(2007, 1, 1, 0, 0),
  'Passengers': 22300,
  'Type': 'Elevated'},
 'Kamppi': {'Name': 'Kamppi',
  'Coordinates': [60.16889, 24.93194],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 30,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1983, 3, 1, 0, 0),
  'Passengers': 39000,
  'Type': 'Underground'},
 'Keilaniemi': {'Name': 'Keilaniemi',
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 20,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 5800,
  'Type': 'Underground'},
 'Koivusaari': {'Name': 'Koivusaari',
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 33,
  'Fare zone': 'A',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 4200,
  'Type': 'Underground'},
 'Kontula': {'Name': 'Kontula',
  'Coordinates': [60.23611, 25.08361],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1986, 10, 21, 0, 0),
  'Passengers': 21400,
  'Type': 'At-grade'},
 'Kulosaari': {'Name': 'Kulosaari',
  'Coordinates': [60.18889, 25.00806],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1982, 6, 1, 0, 0),
  'Passengers': 7100,
  'Type': 'At-grade'},
 'Lauttasaari': {'Name': 'Lauttasaari',
  'Coordinates': [60.15933, 24.87867],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 30,
  'Fare zone': 'A',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 24500,
  'Type': 'Underground'},
 'Matinkylä': {'Name': 'Matinkylä',
  'Coordinates': [60.15944, 24.73833],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 25,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 30500,
  'Type': 'Underground'},
 'Mellunmäki': {'Name': 'Mellunmäki',
  'Coordinates': [60.23917, 25.11083],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1989, 9, 1, 0, 0),
  'Passengers': 14000,
  'Type': 'Elevated'},
 'Myllypuro': {'Name': 'Myllypuro',
  'Coordinates': [60.225, 25.07556],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1986, 10, 21, 0, 0),
  'Passengers': 14000,
  'Type': 'At-grade'},
 'Niittykumpu': {'Name': 'Niittykumpu',
  'Coordinates': [60.17056, 24.76333],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 7600,
  'Type': 'Underground'},
 'Puotila': {'Name': 'Puotila',
  'Coordinates': [60.21472, 25.09306],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1998, 8, 31, 0, 0),
  'Passengers': 10200,
  'Type': 'Underground'},
 'Rautatientori': {'Name': 'Central railway station metro station (Helsinki) - Wikipedia',
  'Coordinates': [60.17056, 24.94056],
  'Tracks': 2,
  'Depth': 27,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1982, 7, 1, 0, 0),
  'Passengers': 76600,
  'Type': 'Underground'},
 'Rastila': {'Name': 'Rastila',
  'Coordinates': [60.20528, 25.12139],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1998, 8, 31, 0, 0),
  'Passengers': 10100,
  'Type': 'At-grade'},
 'Ruoholahti': {'Name': 'Ruoholahti',
  'Coordinates': [60.16306, 24.91444],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1993, 8, 16, 0, 0),
  'Passengers': 32500,
  'Type': 'Underground'},
 'Siilitie': {'Name': 'Siilitie',
  'Coordinates': [60.20528, 25.04361],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1982, 6, 1, 0, 0),
  'Passengers': 9700,
  'Type': 'Elevated'},
 'Sörnäinen': {'Name': 'Sörnäinen',
  'Coordinates': [60.1875, 24.96111],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 25,
  'Fare zone': 'A',
  'Opened': datetime.datetime(1984, 9, 1, 0, 0),
  'Passengers': 56000,
  'Type': 'Underground'},
 'Tapiola': {'Name': 'Tapiola',
  'Coordinates': [60.175, 24.80333],
  'Platforms': 1,
  'Tracks': 2,
  'Depth': 30,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 13200,
  'Type': 'Underground'},
 'Urheilupuisto': {'Name': 'Urheilupuisto',
  'Coordinates': [60.167, 24.783],
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(2017, 11, 18, 0, 0),
  'Passengers': 7000,
  'Type': 'Underground'},
 'Vuosaari': {'Name': 'Vuosaari',
  'Coordinates': [60.20722, 25.14222],
  'Platforms': 1,
  'Tracks': 2,
  'Fare zone': 'B',
  'Opened': datetime.datetime(1998, 8, 31, 0, 0),
  'Passengers': 26800,
  'Type': 'At-grade'}}


df = pd.DataFrame.from_dict(luettelo)
df = df.T
df = df.drop(["Name"], axis=1)
df


df["Platforms"] = df["Platforms"].mask(df["Platforms"].isnull(), 1)
df["Depth"] = df["Depth"].mask(df["Type"]== "At-grade", 0)
df["Depth"] = df["Depth"].mask(df["Type"]== "Elevated", 0)
df


len(stations)

25


stationTypes = df["Type"].value_counts().to_frame()
fig = px.pie(stationTypes, values='Type', names=stationTypes.index, title='Station Types')
fig.show()


#df[df["Type"]=="Elevated"]
df[df["Type"]=="Underground"]
#df[df["Type"]=="At-grade"]


dfTracks = df["Tracks"].value_counts().to_frame()
fig = px.pie(dfTracks, values='Tracks', names = dfTracks.index, title='Number of tracks')
fig.show()


df[df["Tracks"]==3]


dfPlatforms = df["Platforms"].value_counts().to_frame()
fig = px.pie(dfPlatforms,  values='Platforms', names=dfPlatforms.index,  title='Number of platforms')
fig.show()


df[df["Platforms"]==2]


dfFare = df["Fare zone"].value_counts().to_frame()
fig = px.pie(dfFare,  values='Fare zone', names=dfFare.index,  title='Fare zones')
fig.show()


df[df["Fare zone"]=="A"]


dfDepth = df[df["Depth"].notnull()].sort_values(["Depth"], ascending=False)


fig = px.bar(dfDepth, x = dfDepth.index, y='Depth', labels={'index':'stations'})
fig.show()


dfPassenger = df.sort_values(["Passengers"], ascending=False)

fig = px.bar(dfPassenger, x = dfPassenger.index, y='Passengers', labels={'index':'stations'})
fig.show()


dfOpened = df.sort_values(["Opened"])
dfOpened["Year"] = dfOpened["Opened"].dt.year
#dfOpened


c = dfOpened["Year"].value_counts().to_frame()


fig = px.bar(c, x = c.index, y='Year', labels={'Year':'Number of opened stations', "index":"Year"})
fig.show()


import folium


my_map = folium.Map(location = [60.18000, 24.95056], 
                                         zoom_start = 12)


asemat = [key for key in luettelo]
asemat.remove('Aalto-yliopisto')
asemat.remove('Keilaniemi')
asemat.remove('Koivusaari')
#asemat


for asema in asemat:
    folium.Marker(luettelo[asema]['Coordinates'], 
              popup = asema).add_to(my_map)


my_map


#stations


firstLetters = []
allLetters = []
for station in stations:
    station = station.upper()
    firstLetters.append(station[0])
    for letter in station:
        allLetters.append(letter)
#firstLetters
#allLetters


letters = string.ascii_uppercase
letters = letters + "Ä" + "Ö"


#letters


letterdict = {}
for letter in letters:
    letterdict[letter] = 0
letterdictAll = letterdict.copy()


for letter in firstLetters:
    letterdict[letter] +=1
#letterdict


for letter in allLetters:
    if letter in letterdictAll.keys():
        letterdictAll[letter] +=1


dfLetters = pd.DataFrame(letterdict, index=[0])
dfLetters = dfLetters.T
dfLetters.rename(columns = {0:"First letter"}, inplace=True)
#dfLetters


dflettersAll = pd.DataFrame(letterdictAll, index=[0]).T
dflettersAll.rename(columns = {0:"All letters"}, inplace=True)
#dflettersAll


dfL = pd.merge(left=dfLetters, right=dflettersAll, left_index = True, right_index=True)
#dfL


fig = px.bar(dfL, x = dfL.index, y='First letter')
fig.show()


df["Name"] = df.index
df[df["Name"].str.startswith("K")]


fig = px.bar(dfL, x = dfL.index, y='All letters')
fig.show()


dfLsorted = dfL["All letters"].sort_values(ascending=False)

fig = px.bar(dfLsorted, x = dfLsorted.index, y='All letters')
fig.show()


df[df["Name"].str.lower().str.contains("v")]

Web scraping for Helsinki metro stations¶

Scraping the digest page of Helsinki Metro stations¶

Scraping individual web-pages of Helsinki Metro stations. Collecting data¶

Function for automatic scraping the individual pages¶

Function for cleaning the collected data about metro station features¶

Constructing the DataFrame based on the collected data¶

Tidying the DataFrame¶

Some trivia about Helsinki Metro stations¶

Station types¶

Number of tracks in the station¶

Number of platforms on the station¶

Distribution among Fare zones¶

Depth of the stations¶

Daily passengers¶

Timeline of stations opening¶

Placing on map¶

Fun trivia about stations namings¶

	Platforms	Tracks	Depth	Fare zone	Opened	Passengers	Type	Coordinates
Aalto-yliopisto	1	2	20	B	2017-11-18	14500	Underground	NaN
Hakaniemi	1	2	23	A	1982-06-01	28100	Underground	[60.18, 24.95056]
Herttoniemi	1	2	NaN	B	1982-06-01	34500	At-grade	[60.195, 25.03083]
Itäkeskus	2	3	NaN	B	1982-06-01	40600	At-grade	[60.21028, 25.07806]
Helsingin yliopisto	NaN	2	27	A	1995-03-01	24000	Underground	[60.17222, 24.9475]
Kalasatama	2	2	NaN	A	2007-01-01	22300	Elevated	[60.1875, 24.97694]
Kamppi	1	2	30	A	1983-03-01	39000	Underground	[60.16889, 24.93194]
Keilaniemi	1	2	20	B	2017-11-18	5800	Underground	NaN
Koivusaari	1	2	33	A	2017-11-18	4200	Underground	NaN
Kontula	1	2	NaN	B	1986-10-21	21400	At-grade	[60.23611, 25.08361]
Kulosaari	1	2	NaN	A	1982-06-01	7100	At-grade	[60.18889, 25.00806]
Lauttasaari	1	2	30	A	2017-11-18	24500	Underground	[60.15933, 24.87867]
Matinkylä	1	2	25	B	2017-11-18	30500	Underground	[60.15944, 24.73833]
Mellunmäki	1	2	NaN	B	1989-09-01	14000	Elevated	[60.23917, 25.11083]
Myllypuro	1	2	NaN	B	1986-10-21	14000	At-grade	[60.225, 25.07556]
Niittykumpu	1	2	NaN	B	2017-11-18	7600	Underground	[60.17056, 24.76333]
Puotila	1	2	NaN	B	1998-08-31	10200	Underground	[60.21472, 25.09306]
Rautatientori	NaN	2	27	A	1982-07-01	76600	Underground	[60.17056, 24.94056]
Rastila	1	2	NaN	B	1998-08-31	10100	At-grade	[60.20528, 25.12139]
Ruoholahti	1	2	NaN	A	1993-08-16	32500	Underground	[60.16306, 24.91444]
Siilitie	1	2	NaN	B	1982-06-01	9700	Elevated	[60.20528, 25.04361]
Sörnäinen	1	2	25	A	1984-09-01	56000	Underground	[60.1875, 24.96111]
Tapiola	1	2	30	B	2017-11-18	13200	Underground	[60.175, 24.80333]
Urheilupuisto	NaN	2	NaN	B	2017-11-18	7000	Underground	[60.167, 24.783]
Vuosaari	1	2	NaN	B	1998-08-31	26800	At-grade	[60.20722, 25.14222]